{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from simhash import Simhash"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = ['这个程序代码太乱,那个代码规范','这个程序代码不规范,那个更规范','我是佩奇，这是我的弟弟乔治']\n",
    "#先造一个句子"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_items([('这个程序代码太乱', 2), ('那个代码规范', 4), ('这个程序代码不规范', 1), ('那个更规范', 5), ('我是佩奇', 0), ('这是我的弟弟乔治', 3)])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#用TfIdf提取一下特征\n",
    "vec = TfidfVectorizer()\n",
    "D = vec.fit_transform(data)\n",
    "vec.vocabulary_.items()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{2: '这个程序代码太乱', 4: '那个代码规范', 1: '这个程序代码不规范', 5: '那个更规范', 0: '我是佩奇', 3: '这是我的弟弟乔治'}\n"
     ]
    }
   ],
   "source": [
    "voc = dict((i, w) for w, i in vec.vocabulary_.items())\n",
    "print(voc)#用个字典存储一下"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  (0, 4)\t0.7071067811865476\n",
      "  (0, 2)\t0.7071067811865476\n",
      "  (1, 5)\t0.7071067811865476\n",
      "  (1, 1)\t0.7071067811865476\n",
      "  (2, 3)\t0.7071067811865476\n",
      "  (2, 0)\t0.7071067811865476 <class 'scipy.sparse.csr.csr_matrix'> (3, 6) \n",
      " [[0.         0.         0.70710678 0.         0.70710678 0.        ]\n",
      " [0.         0.70710678 0.         0.         0.         0.70710678]\n",
      " [0.70710678 0.         0.         0.70710678 0.         0.        ]]\n"
     ]
    }
   ],
   "source": [
    "print(D,type(D),D.shape,'\\n',D.toarray())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<1x6 sparse matrix of type '<class 'numpy.float64'>'\n",
       "\twith 2 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "D.getrow(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[3 0]\n",
      "[0.70710678 0.70710678]\n"
     ]
    }
   ],
   "source": [
    "print(D.getrow(2).indices)\n",
    "print(D.getrow(2).data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[4 2]\n",
      "[0.70710678 0.70710678]\n",
      "[5 1]\n",
      "[0.70710678 0.70710678]\n",
      "[3 0]\n",
      "[0.70710678 0.70710678]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[<simhash.Simhash at 0x7fbcb8bc3490>,\n",
       " <simhash.Simhash at 0x7fbcb8bba450>,\n",
       " <simhash.Simhash at 0x7fbcb8de0650>]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sh_list = []\n",
    "for i in range(D.shape[0]):\n",
    "    Di = D.getrow(i)\n",
    "    print(Di.indices)\n",
    "    print(Di.data)\n",
    "    #features表示 (token, weight)元祖形式的列表\n",
    "    features = zip([voc[j] for j in Di.indices], Di.data)\n",
    "    sh_list.append(Simhash(features))\n",
    "sh_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23\n",
      "20\n",
      "25\n"
     ]
    }
   ],
   "source": [
    "print(sh_list[0].distance(sh_list[1]))#计算一下相似度\n",
    "print(sh_list[0].distance(sh_list[2]))\n",
    "print(sh_list[1].distance(sh_list[2]))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
